# -*- coding: utf-8 -*-

'''
Created on Sep 8, 2016

@author: hw
'''
import json
import pandas as pd
from _mysql import result
try:
    from urllib.request import urlopen, Request
except ImportError:
    from urllib2 import urlopen, Request 
import pandas as pd
from sqlalchemy import create_engine
import MySQLdb
from settings import DATABASES
from sqlalchemy.sql.sqltypes import *
'''
@attention: 用于从同花顺抓取版块文件，存放到数据库中
'''
class BlockInfo(object):
    '''
    classdocs
    '''
    def __init__(self, totalPage):
        '''
        Constructor
        '''
        self._host = "http://q.10jqka.com.cn"
        self._engin = create_engine(DATABASES["hqdata"].get("engine") + "://" + DATABASES["hqdata"].get("user") + ":" + DATABASES["hqdata"].get("password") + "@" + DATABASES["hqdata"].get("host") + "/" + DATABASES["hqdata"].get("db") + "?charset=utf8") 
        self._totalPage = totalPage
        self._pageSize = 50
    
    '''
    @function: 从同花顺下载版块数据，并存放到数据库表中
    @param retry_count: 重试次数 
    '''     
    def getBlockInfo(self, retry_count=3):
        for _ in range(retry_count):
            try:
                blockcode = []
                blockname = []
                stockcodelist = []
                stocknamelist = []
                stockNum = []
                tasklist={'gn':3,'thshy':1}
                for task in tasklist.keys():
                    for page in range(1, self._totalPage):
                        url = self._host + "/interface/stock/{0}/zdf/desc/{1}/quote/quote".format(task,page)
                        print url
                        request = Request(url)
                        request.add_header("Accept", "application/json,text/javascript,*/*;q=0.01")
                        request.add_header("X-Requested-With", "XMLHttpRequest")
                        request.add_header("Referer", "http://q.10jqka.com.cn/stock/gn")
                        text = urlopen(request, timeout=30).read()
                        content = text.decode()
                        result = json.loads(content)
                        if result["data"] == None:
                            continue
                        for data in result["data"]:
                            num = int(data.get("num"))
                            hycode = data.get("hycode")
                            times = (num + self._pageSize) / self._pageSize
                            platename = data.get("platename")
                            if platename == None:
                                print result["data"]
                                raise IOError('platename is not full{0}'.format(url))
                            for spage in range(1, times + 1): 
                                suburl = self._host + "/interface/stock/detail/zdf/desc/{0}/{1}/{2}".format(spage,tasklist.get(task), hycode)
                                subRequest = Request(suburl)
                                subReferer = self._host + "/stock/gn/{}".format(hycode)
                                subRequest.add_header("Accept", "application/json,text/javascript,*/*;q=0.01")
                                subRequest.add_header("X-Requested-With", "XMLHttpRequest")
                                subRequest.add_header("Referer", subReferer)
                                subReq = urlopen(subRequest, timeout=30).read()
                                subContent = subReq.decode()
                                subResult = json.loads(subContent)
                                for subdata in subResult["data"]:
                                    stockname = subdata.get("stockname")
                                    stockcode = subdata.get("stockcode")
                                    if stockcode.startswith("6"):
                                        stockcode = stockcode + ".XSHG"
                                    else:
                                        stockcode = stockcode + ".XSHE"
                                    blockcode.append(data.get("platecode"))
                                    blockname.append(data.get("platename"))
                                    stockNum.append(num)
                                    stockcodelist.append(stockcode)
                                    stocknamelist.append(stockname)
                
                # 构建dataframe，存放到数据库中    
                dfData = {"blockname":blockname, "num":stockNum, "stockname":stocknamelist}
                df = pd.DataFrame(data=dfData, index=[blockcode, stockcodelist])
                df.index.names = ['blockcode', 'stockcode']
                dttype = {'blockcode':VARCHAR(32), 'blockname':VARCHAR(32), 'stockcode':VARCHAR(32), 'stockname':VARCHAR(32)}
                df.to_sql('up_block_info', self._engin, if_exists='replace', dtype=dttype)
                print "Success,Congratulations to you  !"
                return 
            except Exception as e:
                print(e)


    '''
    @function: 从同花顺下载每日涨停复盘数据，并存放到数据库表中
    @param retry_count: 重试次数 
    '''     
    def getDailyZtData(self, date, retry_count=3):
        url = "http://zx.10jqka.com.cn/event/api/getharden/date/{0}/orderby/code/orderway/asc/charset/GBK/".format(date)
        for _ in range(retry_count):
            try:
                stockcode = []
                date = []
                reason = []
                request = Request(url)
                request.add_header("Accept", "text/html,application/xhtml+xml,text/javascript,*/*;q=0.01")
                request.add_header("X-Requested-With", "XMLHttpRequest")
                request.add_header("Referer", "http://q.10jqka.com.cn/stock/gn")
                text = urlopen(request, timeout=30).read()
                content = text.decode()
                result = json.loads(content)
                for stock in result["data"]:
                    if stock["reason"] == u"新股":
                        continue
                    stockcode.append(stock["code"])
                    reason.append(stock["reason"])
                    date.append(stock["date"])
                df = pd.DataFrame(data={"stockcode":stockcode, "reason":reason, "date":date}, index=date)
                print df
                return
            except Exception as e:
                print(e)

    '''
    @function: 从同花顺下载每日涨停复盘数据，并存放到数据库表中
    @param retry_count: 重试次数 
    '''
    def getDailyZtDataDetail(self, retry_count=3):
#         url = "%22http%3a%2f%2fwww.iwencai.com%2frapid%2fsearch%3fsn%3d1%26qs%3dclient_apptg%26W%3d%e4%bb%8a%e5%a4%a9%e6%b6%a8%e5%81%9c%e7%9a%84%e8%82%a1%e7%a5%a8%ef%bc%8c%e6%b6%a8%e5%81%9c%e7%9a%84%e6%97%b6%e9%97%b4%22"
        import sys
        defaultencoding = 'utf-8'
        if sys.getdefaultencoding() != defaultencoding:
            reload(sys)
            sys.setdefaultencoding(defaultencoding)
        url = "http://www.iwencai.com/stockpick/search?&w=%E4%BB%8A%E5%A4%A9%E6%B6%A8%E5%81%9C%E7%9A%84%E8%82%A1%E7%A5%A8%EF%BC%8C%E6%B6%A8%E5%81%9C%E7%9A%84%E6%97%B6%E9%97%B4&sn=1"
        for _ in range(retry_count):
            try:
                print url
                request = Request(url)
                request.add_header("Accept", "text/html,application/xhtml+xml,text/javascript,*/*;q=0.9,image/webp,*/*;q=0.8")
                request.add_header("User-Agent", "Mozilla/5.0(Windows NT6.1; WOW64) AppleWebKit/537.36 (KHTML,like Gecko) Chrome/45.0.2454.101 Safari/637.36")
                request.add_header("Cookie", "guideState=1;PHPSESSID=bftobv8do7de8k9ov097sibdh6;cid=2tccoc7pjneg5qttqq7pjneg5qttqq70u2dq911474189715;ComputerID=2tccoc7pjneg5qttqq70u2dq91160918+170835")
                text = urlopen(request, timeout=30).read()
                content = text.decode()
                results = content.split("\n")
                for result in results:
                    if result.find("var allResult = ") != -1:
                        import time
                        strJson = result[16:-1]
                        tmpObj = json.loads(strJson)
                        stkcodelist = []  # 股票代码
                        stknamelist = []  # 名称
                        marketlist = []  # 市场代码
                        datelist = []  # 涨停日期
                        firsttimelist = []  # 第一次涨停时间
                        lasttimelist = []  # 最终涨停时间
                        lxztdaynumlist = []  # 连续涨停天数
                        reasonlist = []  # 涨停原因
                        opennumlist = []  # 涨停开板次数
                        for zt in tmpObj["result"]:
                            if zt[9] == u"新股":
                                continue
                            codeinfo = zt[0].split(".")
                            code = codeinfo[0] + ".XSHG" if (zt[0].endswith(".SH")) else codeinfo[0] + ".XSHE"
                            market = "XSHG" if (zt[0].endswith(".SH")) else "XSHE"
                            stkcodelist.append(code)
                            stknamelist.append(str(zt[1]))
                            marketlist.append(market)
                            datelist.append(time.strftime('%Y%m%d', time.localtime(int(zt[5]) / 1000)))
                            firsttimelist.append(time.strftime('%H:%M:%S', time.localtime(int(zt[5]) / 1000)))
                            lasttimelist.append(time.strftime('%H:%M:%S', time.localtime(int(zt[6]) / 1000)))
                            lxztdaynumlist.append(int(zt[8]))
                            reasonlist.append(zt[9])
                            opennumlist.append(zt[14])
                        data = {"stkcode":stkcodelist, 
                                "stkname":stknamelist, 
                                "market":marketlist, 
                                "date":datelist, 
                                "firsttime":firsttimelist, 
                                "lasttime":lasttimelist, 
                                "daycount":lxztdaynumlist, 
                                "reason":reasonlist, 
                                "opennum":opennumlist
                                }
                        # 构建dataframe，存放到数据库中    
                        df = pd.DataFrame(data=data)
                        df.index.name='id'
                        dttype = {'stkcode':VARCHAR(32), 'stkname':VARCHAR(32), 'market':VARCHAR(32), 'date':DATE,'reason':VARCHAR(32),'opennum':INT,'firsttime':TIME,'lasttime':TIME}
                        df.to_sql('stock_dailylimit_info', self._engin, if_exists='append', dtype=dttype,index=False)
                        print "Success,Congratulations to you  !"                        
                return
            except Exception as e:
                print(e)

if __name__ == '__main__':
    blockinfo = BlockInfo(6)
    blockinfo.getBlockInfo(retry_count=3)
#     blockinfo.getDailyZtDataDetail(retry_count=3)
#     blockinfo.getDailyZtData(date='2016-09-14',retry_count=3)

